<!DOCTYPE html>
<html lang="zh-cn">
<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
    <title>Node.js 种子下载器 - 早起的年轻人</title>
    <meta name="renderer" content="webkit" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>

<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />

<meta name="theme-color" content="#f8f5ec" />
<meta name="msapplication-navbutton-color" content="#f8f5ec">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="#f8f5ec">


<meta name="author" content="早起的年轻人" /><meta name="description" content="庆祝 2018 国庆，制作了一个 Node.js 的种子下载器。爬取页面，根据页面的链接，破解另外一个网站，下载种子文件。项目比较简单，爬取页面没有使用任何爬虫框架。项目源码
" /><meta name="keywords" content="管理, Github, Vue, flutter, Go" />






<meta name="generator" content="Hugo 0.79.1 with even 4.0.0" />


<link rel="canonical" href="https://luckly.work/post/2018/2018-10-11-nodejs-seed-downloader/" />
<link href="/post/2018/2018-10-11-nodejs-seed-downloader/" rel="alternate" type="application/rss+xml" title="早起的年轻人" />
<link href="/post/2018/2018-10-11-nodejs-seed-downloader/" rel="feed" type="application/rss+xml" title="早起的年轻人" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/manifest.json">
<link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">

<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
<link href="/dist/even.06658218.min.css" rel="stylesheet">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@3.1.20/dist/jquery.fancybox.min.css" integrity="sha256-7TyXnr2YU040zfSP+rEcz29ggW4j56/ujTPwjMzyqFY=" crossorigin="anonymous">
<link rel="stylesheet" href="/css/reset-even.css">


<meta property="og:title" content="Node.js 种子下载器" />
<meta property="og:description" content="庆祝 2018 国庆，制作了一个 Node.js 的种子下载器。爬取页面，根据页面的链接，破解另外一个网站，下载种子文件。项目比较简单，爬取页面没有使用任何爬虫框架。项目源码" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://luckly.work/post/2018/2018-10-11-nodejs-seed-downloader/" />
<meta property="article:published_time" content="2018-10-11T12:58:44+08:00" />
<meta property="article:modified_time" content="2018-10-11T12:58:44+08:00" />
<meta itemprop="name" content="Node.js 种子下载器">
<meta itemprop="description" content="庆祝 2018 国庆，制作了一个 Node.js 的种子下载器。爬取页面，根据页面的链接，破解另外一个网站，下载种子文件。项目比较简单，爬取页面没有使用任何爬虫框架。项目源码">
<meta itemprop="datePublished" content="2018-10-11T12:58:44+08:00" />
<meta itemprop="dateModified" content="2018-10-11T12:58:44+08:00" />
<meta itemprop="wordCount" content="1842">



<meta itemprop="keywords" content="Spider,Node.js," />
<meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="Node.js 种子下载器"/>
<meta name="twitter:description" content="庆祝 2018 国庆，制作了一个 Node.js 的种子下载器。爬取页面，根据页面的链接，破解另外一个网站，下载种子文件。项目比较简单，爬取页面没有使用任何爬虫框架。项目源码"/>

</head>
<body>
<div id="mobile-navbar" class="mobile-navbar">
    <div class="mobile-header-logo">
        <a href="/" class="logo">早起的年轻人</a>
    </div>
    <div class="mobile-navbar-icon">
        <span></span>
        <span></span>
        <span></span>
    </div>
</div>
<nav id="mobile-menu" class="mobile-menu slideout-menu">
    <ul class="mobile-menu-list">
        <a href="/">
            <li class="mobile-menu-item" title="首页">首页</li>
        </a>
        <a href="/post/">
            <li class="mobile-menu-item" title="归档">归档</li>
        </a>
        <a href="/categories/">
            <li class="mobile-menu-item" title="分类">分类</li>
        </a>
        <a href="/tags/">
            <li class="mobile-menu-item" title="标签">标签</li>
        </a>
        <a href="/about/">
            <li class="mobile-menu-item" title="关于我">关于我</li>
        </a>
        <a href="/friend/">
            <li class="mobile-menu-item" title="友链">友链</li>
        </a>
        <a href="https://github.com/ITmxs/">
            <li class="mobile-menu-item" title="阅读清单">阅读清单</li>
        </a>
        
    </ul>
    <form class="search-form" style="position: absolute;">
    <input name="search" type="search" style="height: 28px;padding-left: 30px;"/>
    <input type="image" src="/img/search.svg" alt="Search" style="position: absolute;left: 6px;top: 6px;"/>
</form>


</nav>

<div class="container" id="mobile-panel">
    <header id="header" class="header">
        <div class="logo-wrapper">
  <a href="/%20/" class="logo">早起的年轻人</a>
</div>

<nav class="site-navbar">
  <ul id="menu" class="menu">
    <li class="menu-item">
      <a class="menu-item-link" href="/">首页</a>
    </li><li class="menu-item">
      <a class="menu-item-link" href="/post/">归档</a>
    </li><li class="menu-item">
      <a class="menu-item-link" href="/categories/">分类</a>
    </li><li class="menu-item">
      <a class="menu-item-link" href="/tags/">标签</a>
    </li><li class="menu-item">
      <a class="menu-item-link" href="/about/">关于我</a>
    </li><li class="menu-item">
      <a class="menu-item-link" href="/friend/">友链</a>
    </li><li class="menu-item">
      <a class="menu-item-link" href="https://github.com/ITmxs/">阅读清单</a>
    </li>
  </ul>
</nav>
        <form class="search-form" style="position: absolute;">
    <input name="search" type="search" style="height: 28px;padding-left: 30px;"/>
    <input type="image" src="/img/search.svg" alt="Search" style="position: absolute;left: 6px;top: 6px;"/>
</form>



    </header>


    <main id="main" class="main">
        <div class="content-wrapper">
            <div id="content" class="content">
                <article class="post">
  
  <header class="post-header">
    <h1 class="post-title">Node.js 种子下载器</h1>

    <div class="post-meta">
      <span class="post-time"> 2018-10-11 12:58 </span>
      <div class="post-category">
        <a href="/categories/%E6%8A%80%E6%9C%AF/"> 技术 </a>
        </div>
      <span class="more-meta"> 约 1842 字 </span>
      <span class="more-meta"> 预计阅读 4 分钟 </span>
      <span id="busuanzi_container_page_pv" class="more-meta"> <span id="busuanzi_value_page_pv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 次阅读 </span>
    </div>
  </header>

  <div class="post-toc" id="post-toc">
  <h2 class="post-toc-title">文章目录</h2>
  <div class="post-toc-content">
    <nav id="TableOfContents">
  <ul>
    <li>
      <ul>
        <li><a href="#项目初始化">项目初始化</a></li>
        <li><a href="#安装依赖包">安装依赖包</a></li>
        <li><a href="#爬取网页策略">爬取网页策略</a></li>
        <li><a href="#请求代理">请求代理</a></li>
        <li><a href="#解决中文乱码的问题">解决中文乱码的问题</a></li>
        <li><a href="#并发请求">并发请求</a></li>
        <li><a href="#图片下载">图片下载</a></li>
        <li><a href="#破解网站种子下载">破解网站种子下载</a></li>
        <li><a href="#面向对象">面向对象</a></li>
        <li><a href="#总结">总结</a></li>
      </ul>
    </li>
  </ul>
</nav>
  </div>
</div>
  <div class="post-outdated">
    <div class="warn">
      <p>【注意】最后更新于 <span class="timeago" datetime="2018-10-11T12:58:44" title="October 11, 2018">October 11, 2018</span>，文中内容可能已过时，请谨慎使用。</p>
    </div>
  </div>
  <div class="post-content">
    <p>庆祝 2018 国庆，制作了一个 <code>Node.js</code> 的种子下载器。爬取页面，根据页面的链接，破解另外一个网站，下载种子文件。项目比较简单，爬取页面没有使用任何爬虫框架。<a href="https://github.com/nusr/FBIWarning">项目源码</a></p>
<p><code>Node.js</code> 的安装请看我的另外一篇文章，<a href="https://www.cnblogs.com/stevexu/p/9734249.html">Node.js 的多版本安装</a>。</p>
<h2 id="项目初始化">项目初始化</h2>
<p>新建一个文件夹 <code>FBIWarning</code>,在该文件夹下打开命令行 <code>CMD</code> 或者 <code>git bash</code>。运行 <code>npm init -y</code>,该文件夹会生成一个 <code>package.json</code> 文件。</p>
<h2 id="安装依赖包">安装依赖包</h2>
<p>安装依赖包 <code>cnpm install --save cheerio iconv-lite request socks5-http-client</code>。每个依赖包的功能如下：</p>
<ul>
<li><code>cheerio</code> // 解析 DOM</li>
<li><code>iconv-lite</code> // 解决中文乱码的问题</li>
<li><code>request</code> // http 请求，图片和种子的下载</li>
<li><code>socks5-http-client</code> // socks 代理</li>
</ul>
<h2 id="爬取网页策略">爬取网页策略</h2>
<p>网页之间，是靠链接联系在一起的，符合数据结构里面的图状结构。所以，对应有如下两种爬取策略。</p>
<ol>
<li>爬取所有列表页面的链接后，再去爬取所有详情页面，对应图算法的广度优先遍历。</li>
<li>爬取一部分列表页面，就去爬取详情页面。然后再去爬取列表页面，爬取详情页面，循环进行，对应图算法的深度优先遍历。</li>
</ol>
<p>因为是国外网站，网络可能随时断开，所以采用第二种策略比较好。同时，也能很快得到种子文件。为了防止重复爬取页面，可以将爬取页面的链接作为索引。</p>
<h2 id="请求代理">请求代理</h2>
<p>网站是国外网站，需要使用梯子，否则不能爬取。<a href="https://github.com/nusr/FBIWarning">代理传送门</a>。<code>socks5-http-client</code> 配合 <code>reqeust</code> 使用，可以解决代理的问题。但是，该代理只支持 <code>socks</code> 代理, <code>http(s)</code> 代理暂不支持。</p>
<h2 id="解决中文乱码的问题">解决中文乱码的问题</h2>
<p>目标网站的页面编码是 <code>gbk</code> ，而 <code>request</code> 依赖包的默认编码是 <code>UTF-8</code>,使用默认编码解码方式，会导致页面的中文变成乱码。所以得到返回数据前，去掉默认编码，就是设置编码为 <code>encoding: null</code>,然后使用 <code>iconv-lite</code> 使用 <code>gbk</code> 方式解码，这样就可以解决中文编码为乱码的问题，代码如下：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-js" data-lang="js"><span class="kr">const</span> <span class="nx">request</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s2">&#34;request&#34;</span><span class="p">);</span>
<span class="c1">// 解析 dom
</span><span class="c1"></span><span class="kr">const</span> <span class="nx">cheerio</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s2">&#34;cheerio&#34;</span><span class="p">);</span>
<span class="c1">// 中文编码
</span><span class="c1"></span><span class="kr">const</span> <span class="nx">iconv</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s2">&#34;iconv-lite&#34;</span><span class="p">);</span>
<span class="c1">// 代理
</span><span class="c1"></span><span class="kr">const</span> <span class="nx">Agent</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s2">&#34;socks5-http-client/lib/Agent&#34;</span><span class="p">);</span>
<span class="kr">const</span> <span class="nx">COMMON_CONFIG</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="s2">&#34;./config&#34;</span><span class="p">);</span>
<span class="cm">/**
</span><span class="cm"> * 请求页面
</span><span class="cm"> * @param {String} requestUrl 请求页面
</span><span class="cm"> */</span>
<span class="kd">function</span> <span class="nx">requestPage</span><span class="p">(</span><span class="nx">requestUrl</span><span class="p">)</span> <span class="p">{</span>
  <span class="k">try</span> <span class="p">{</span>
    <span class="k">return</span> <span class="k">new</span> <span class="nb">Promise</span><span class="p">((</span><span class="nx">resolve</span><span class="p">,</span> <span class="nx">reject</span><span class="p">)</span> <span class="p">=&gt;</span> <span class="p">{</span>
      <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">requestUrl</span><span class="p">)</span> <span class="p">{</span>
        <span class="nx">resolve</span><span class="p">(</span><span class="kc">false</span><span class="p">);</span>
      <span class="p">}</span>
      <span class="nx">request</span><span class="p">.</span><span class="nx">get</span><span class="p">(</span>
        <span class="p">{</span>
          <span class="nx">url</span><span class="o">:</span> <span class="nx">requestUrl</span><span class="p">,</span>
          <span class="nx">agentClass</span><span class="o">:</span> <span class="nx">Agent</span><span class="p">,</span>
          <span class="nx">agentOptions</span><span class="o">:</span> <span class="p">{</span>
            <span class="nx">socksPort</span><span class="o">:</span> <span class="mi">13838</span><span class="p">,</span> <span class="c1">// 代理端口
</span><span class="c1"></span>            <span class="nx">socksHost</span><span class="o">:</span> <span class="s2">&#34;127.0.0.1&#34;</span> <span class="c1">// 代理 Host
</span><span class="c1"></span>          <span class="p">},</span>
          <span class="nx">headers</span><span class="o">:</span> <span class="p">{</span>
            <span class="s2">&#34;User-Agent&#34;</span><span class="o">:</span>
              <span class="s2">&#34;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36&#34;</span>
          <span class="p">},</span>
          <span class="c1">// 去掉默认 utf-8 解码，否则解码会乱码
</span><span class="c1"></span>          <span class="nx">encoding</span><span class="o">:</span> <span class="kc">null</span>
        <span class="p">},</span>
        <span class="kd">function</span><span class="p">(</span><span class="nx">err</span><span class="p">,</span> <span class="nx">response</span><span class="p">,</span> <span class="nx">body</span><span class="p">)</span> <span class="p">{</span>
          <span class="c1">// 防止解析报错
</span><span class="c1"></span>          <span class="k">try</span> <span class="p">{</span>
            <span class="c1">// 统一解决中文乱码的问题
</span><span class="c1"></span>            <span class="kd">let</span> <span class="nx">content</span> <span class="o">=</span> <span class="nx">iconv</span><span class="p">.</span><span class="nx">decode</span><span class="p">(</span><span class="nx">body</span><span class="p">,</span> <span class="s2">&#34;gbk&#34;</span><span class="p">);</span>
            <span class="kd">let</span> <span class="nx">$</span> <span class="o">=</span> <span class="nx">cheerio</span><span class="p">.</span><span class="nx">load</span><span class="p">(</span><span class="nx">content</span><span class="p">);</span>
            <span class="nx">resolve</span><span class="p">(</span><span class="nx">$</span><span class="p">,</span> <span class="nx">err</span><span class="p">,</span> <span class="nx">response</span><span class="p">,</span> <span class="nx">body</span><span class="p">,</span> <span class="nx">content</span><span class="p">);</span>
          <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="nx">error</span><span class="p">)</span> <span class="p">{</span>
            <span class="nx">resolve</span><span class="p">(</span><span class="kc">null</span><span class="p">);</span>
          <span class="p">}</span>
        <span class="p">}</span>
      <span class="p">);</span>
    <span class="p">});</span>
  <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="nx">error</span><span class="p">)</span> <span class="p">{</span>
    <span class="c1">//如果连续发出多个请求，即使某个请求失败，也不影响后面的其他请求
</span><span class="c1"></span>    <span class="nb">Promise</span><span class="p">.</span><span class="nx">resolve</span><span class="p">(</span><span class="kc">null</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="并发请求">并发请求</h2>
<p>分页请求有很多个，可以使用递归来一个一个请求，但是写法不太好看。所以，可以使用 <code>ES7+</code> 里面的 <code>async</code> 函数，将同步过程变为异步过程。<code>async</code> 要配合 <code>await</code> 使用，就可以将同步过程变为异步过程。详细了解 <code>async</code> 请看<a href="http://es6.ruanyifeng.com/#docs/async">阮一峰 ES async</a></p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-js" data-lang="js"><span class="kr">async</span> <span class="kd">function</span> <span class="nx">innerRecursion</span><span class="p">()</span> <span class="p">{</span>
  <span class="k">for</span> <span class="p">(</span><span class="kd">let</span> <span class="nx">i</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span> <span class="nx">i</span> <span class="o">&lt;=</span> <span class="mi">100</span><span class="p">;</span> <span class="nx">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span>
    <span class="kd">let</span> <span class="nx">requestUrl</span> <span class="o">=</span> <span class="s2">&#34;http://www.baidu.com?page=&#34;</span> <span class="o">+</span> <span class="nx">i</span><span class="p">;</span> <span class="c1">// 事例网站，非爬取网站
</span><span class="c1"></span>    <span class="kd">let</span> <span class="nx">result</span> <span class="o">=</span> <span class="kr">await</span> <span class="k">this</span><span class="p">.</span><span class="nx">requestPage</span><span class="p">(</span><span class="nx">url</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</code></pre></td></tr></table>
</div>
</div><p>一个一个请求比较慢，可以使用 <code>Promise.all</code> 实现并发请求。当然，也可以使用 <code>async</code> <a href="https://github.com/caolan/async">模块</a> 提高下载的并发量，有需要的可以自己去了解。这个 <code>async</code> 模块并非上面的 <code>async</code> 函数。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-js" data-lang="js"><span class="kd">function</span> <span class="nx">innerRecursion</span><span class="p">()</span> <span class="p">{</span>
  <span class="kd">let</span> <span class="nx">requestUrls</span> <span class="o">=</span> <span class="p">[];</span>
  <span class="k">for</span> <span class="p">(</span><span class="kd">let</span> <span class="nx">i</span> <span class="o">=</span> <span class="mi">1</span><span class="p">;</span> <span class="nx">i</span> <span class="o">&lt;=</span> <span class="mi">100</span><span class="p">;</span> <span class="nx">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span>
    <span class="kd">let</span> <span class="nx">requestUrl</span> <span class="o">=</span> <span class="s2">&#34;http://www.baidu.com?page=&#34;</span> <span class="o">+</span> <span class="nx">i</span><span class="p">;</span> <span class="c1">// 事例网站，非爬取网站
</span><span class="c1"></span>    <span class="nx">requestUrls</span><span class="p">.</span><span class="nx">push</span><span class="p">(</span><span class="nx">requestUrl</span><span class="p">);</span>
  <span class="p">}</span>
  <span class="kd">let</span> <span class="nx">promises</span> <span class="o">=</span> <span class="nx">requestUrls</span><span class="p">.</span><span class="nx">map</span><span class="p">(</span><span class="nx">url</span> <span class="p">=&gt;</span> <span class="k">this</span><span class="p">.</span><span class="nx">requestPage</span><span class="p">(</span><span class="nx">url</span><span class="p">));</span>
  <span class="nb">Promise</span><span class="p">.</span><span class="nx">all</span><span class="p">(</span><span class="nx">promises</span><span class="p">)</span>
    <span class="p">.</span><span class="nx">then</span><span class="p">(</span><span class="nx">results</span> <span class="p">=&gt;</span> <span class="p">{</span>
      <span class="c1">// results 是一个数组，对应上面每个请求的结果
</span><span class="c1"></span>    <span class="p">})</span>
    <span class="p">.</span><span class="k">catch</span><span class="p">(</span><span class="nx">error</span> <span class="p">=&gt;</span> <span class="p">{</span>
      <span class="c1">// 捕获请求中可能发生的错误
</span><span class="c1"></span>      <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="nx">error</span><span class="p">);</span>
    <span class="p">});</span>
<span class="p">}</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="图片下载">图片下载</h2>
<p>图片的下载非常简单,代码如下：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-js" data-lang="js"><span class="cm">/**
</span><span class="cm"> * 下载文件
</span><span class="cm"> * @param {String} url 请求链接
</span><span class="cm"> * @param {String} filePath 文件路径
</span><span class="cm"> */</span>
<span class="kd">function</span> <span class="nx">downloadFile</span><span class="p">(</span><span class="nx">url</span><span class="p">,</span> <span class="nx">filePath</span><span class="p">)</span> <span class="p">{</span>
  <span class="c1">// try...catch 防止一个请求出错，导致程序终止 种子的下载相同
</span><span class="c1"></span>  <span class="k">try</span> <span class="p">{</span>
    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">url</span> <span class="o">||</span> <span class="o">!</span><span class="nx">filePath</span><span class="p">)</span> <span class="p">{</span>
      <span class="k">return</span> <span class="kc">false</span><span class="p">;</span>
    <span class="p">}</span>
    <span class="nx">request</span>
      <span class="p">.</span><span class="nx">get</span><span class="p">({</span>
        <span class="nx">url</span><span class="p">,</span>
        <span class="nx">agentClass</span><span class="o">:</span> <span class="nx">Agent</span><span class="p">,</span>
        <span class="nx">agentOptions</span><span class="o">:</span> <span class="p">{</span>
          <span class="nx">socksPort</span><span class="o">:</span> <span class="mi">13838</span><span class="p">,</span> <span class="c1">// 代理端口
</span><span class="c1"></span>          <span class="nx">socksHost</span><span class="o">:</span> <span class="s2">&#34;127.0.0.1&#34;</span> <span class="c1">// 代理 Host
</span><span class="c1"></span>        <span class="p">},</span>
        <span class="nx">headers</span><span class="o">:</span> <span class="p">{</span>
          <span class="nx">headers</span><span class="o">:</span> <span class="p">{</span>
            <span class="s2">&#34;User-Agent&#34;</span><span class="o">:</span>
              <span class="s2">&#34;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36&#34;</span>
          <span class="p">}</span>
        <span class="p">}</span>
      <span class="p">})</span>
      <span class="p">.</span><span class="nx">pipe</span><span class="p">(</span><span class="nx">fs</span><span class="p">.</span><span class="nx">createWriteStream</span><span class="p">(</span><span class="nx">filePath</span><span class="p">));</span>
  <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="nx">error</span><span class="p">)</span> <span class="p">{</span>
    <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="nx">error</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="破解网站种子下载">破解网站种子下载</h2>
<p>解析详情页面，只能得到类似 <code>http://www.jandown.com?ref=VENU794</code> 的链接，需要破解该网站的种子下载。查看网站的种子下载方式，就是一个 <code>post</code> 请求，后端就会返回种子文件。刚开始的时候，不熟悉服务端的表单提交方式，导致文件一直得不到，后来详细查看了 <code>request</code> 的官文文档，发现是自己写错了。结合上面的图片下载，种子的下载方式自然就有了，代码如下：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-js" data-lang="js"><span class="cm">/**
</span><span class="cm"> * 下载种子链接
</span><span class="cm"> * @param {String} childDir // 子目录
</span><span class="cm"> * @param {String} downloadUrl  // 下载种子地址
</span><span class="cm"> */</span>
<span class="kd">function</span> <span class="nx">downloadTorrent</span><span class="p">(</span><span class="nx">childDir</span><span class="p">,</span> <span class="nx">downloadUrl</span><span class="p">)</span> <span class="p">{</span>
  <span class="k">try</span> <span class="p">{</span>
    <span class="c1">// 解析出链接的 code 值
</span><span class="c1"></span>    <span class="kd">let</span> <span class="nx">code</span> <span class="o">=</span> <span class="nx">querystring</span><span class="p">.</span><span class="nx">parse</span><span class="p">(</span><span class="nx">downloadUrl</span><span class="p">.</span><span class="nx">split</span><span class="p">(</span><span class="s2">&#34;?&#34;</span><span class="p">).</span><span class="nx">pop</span><span class="p">()).</span><span class="nx">ref</span><span class="p">;</span>
    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">code</span> <span class="o">||</span> <span class="o">!</span><span class="nx">childDir</span><span class="p">)</span> <span class="p">{</span>
      <span class="k">return</span> <span class="kc">false</span><span class="p">;</span>
    <span class="p">}</span>
    <span class="c1">// 发出 post 请求，然后接受文件即可
</span><span class="c1"></span>    <span class="nx">request</span>
      <span class="p">.</span><span class="nx">post</span><span class="p">({</span>
        <span class="nx">url</span><span class="o">:</span> <span class="s2">&#34;http://www.jandown.com/fetch.php&#34;</span><span class="p">,</span>
        <span class="nx">agentClass</span><span class="o">:</span> <span class="nx">Agent</span><span class="p">,</span>
        <span class="nx">agentOptions</span><span class="o">:</span> <span class="p">{</span>
          <span class="nx">socksPort</span><span class="o">:</span> <span class="mi">13838</span><span class="p">,</span> <span class="c1">// 代理端口
</span><span class="c1"></span>          <span class="nx">socksHost</span><span class="o">:</span> <span class="s2">&#34;127.0.0.1&#34;</span> <span class="c1">// 代理 Host
</span><span class="c1"></span>        <span class="p">},</span>
        <span class="nx">headers</span><span class="o">:</span> <span class="p">{</span>
          <span class="s2">&#34;User-Agent&#34;</span><span class="o">:</span>
            <span class="s2">&#34;Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36&#34;</span>
        <span class="p">},</span>
        <span class="nx">formData</span><span class="o">:</span> <span class="p">{</span>
          <span class="nx">code</span>
        <span class="p">}</span>
      <span class="p">})</span>
      <span class="p">.</span><span class="nx">pipe</span><span class="p">(</span><span class="nx">fs</span><span class="p">.</span><span class="nx">createWriteStream</span><span class="p">(</span><span class="nx">childDir</span> <span class="o">+</span> <span class="s2">&#34;/&#34;</span> <span class="o">+</span> <span class="nx">code</span> <span class="o">+</span> <span class="s2">&#34;.torrent&#34;</span><span class="p">));</span>
  <span class="p">}</span> <span class="k">catch</span> <span class="p">(</span><span class="nx">error</span><span class="p">)</span> <span class="p">{</span>
    <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="nx">error</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">}</span>
</code></pre></td></tr></table>
</div>
</div><h2 id="面向对象">面向对象</h2>
<p>刚开始是使用面向过程的方式写的，后来发现代码太重复了，所以采用 <code>OOP</code> 改写了整个代码。详细了解 <code>javaScript Class</code> 请看<a href="http://es6.ruanyifeng.com/#docs/class">阮一峰 ES class</a></p>
<h2 id="总结">总结</h2>
<ol>
<li>学习中文编码为乱码的解决方法</li>
<li>学习了 <code>request</code> 的代理以及文件下载功能</li>
<li>破解种子网站的种子下载功能</li>
<li>js 面向对象开发</li>
<li>爬虫并发量解决</li>
</ol>
<p>感谢阅读！</p>
  </div>

  <div class="post-copyright">
  <p class="copyright-item">
    <span class="item-title">文章作者</span>
    <span class="item-content">早起的年轻人</span>
  </p>
  <p class="copyright-item">
    <span class="item-title">上次更新</span>
    <span class="item-content">
        2018-10-11 12:58
        
    </span>
  </p>
  
  <p class="copyright-item">
    <span class="item-title">许可协议</span>
    <span class="item-content"><a target="_blank" rel="license noopener" href="https://github.com/nusr/blog/blob/master/LICENSE">MIT</a></span>
  </p>
</div>
<div class="post-reward">
  <input type="checkbox" name="reward" id="reward" hidden />
  <label class="reward-button" for="reward">赞赏支持</label>
  <div class="qr-code">
    
    <label class="qr-code-image" for="reward">
        <img class="image" src="/images/wechat.jpg">
        <span>微信佛系打赏</span>
      </label>
    <label class="qr-code-image" for="reward">
        <img class="image" src="/images/alipay.jpg">
        <span>支付宝佛系打赏</span>
      </label>
  </div>
</div><footer class="post-footer">
    <div class="post-tags">
      <a href="/tags/Spider/">Spider</a>
      <a href="/tags/Node.js/">Node.js</a>
      </div>
    <nav class="post-nav">
      <a class="prev" href="/post/2019/2019-01-31-why-I-recommend-you-to-use-kindle/">
        <i class="iconfont icon-left"></i>
        <span class="prev-text nav-default">我为什么推荐你使用kindle</span>
        <span class="prev-text nav-mobile">上一篇</span>
      </a>
      <a class="next" href="/post/2018/2018-10-01-install-multi-version-nodejs/">
        <span class="next-text nav-default">Node.js 多版本安装</span>
        <span class="next-text nav-mobile">下一篇</span>
        <i class="iconfont icon-right"></i>
      </a>
    </nav>
  </footer>
  
</article>
            </div>
            

        </div>
    </main>

    <footer id="footer" class="footer">
        <div class="social-links">
  <a href="852851198@qq.com" class="iconfont icon-email" title="email" rel="noopener" target="_blank" data-title="email"> </a>
  <a href="https://github.com/ITmxs" class="iconfont icon-github" title="github" rel="noopener" target="_blank" data-title="github"> </a>
  <a href="https://juejin.cn/user/3843548384077192" class="iconfont icon-juejin" title="juejin" rel="noopener" target="_blank" data-title="juejin"> </a>
  <a href="https://www.zhihu.com/people/yimi-yang-guang-96-65" class="iconfont icon-zhihu" title="zhihu" rel="noopener" target="_blank" data-title="zhihu"> </a>
  <a href="https://space.bilibili.com/480883651" class="iconfont icon-bilibili" title="bilibili" rel="noopener" target="_blank" data-title="bilibili"> </a>
  <a href="https://luckly.work/index.xml" type="application/rss+xml" rel="noopener" target="_blank" class="iconfont icon-rss" title="rss"></a>


</div>

<div class="copyright">
  <span class="power-by">
    由 <a class="hexo-link" href="https://gohugo.io">Hugo</a> 强力驱动
  </span>
  <span class="division">|</span>
  <span class="theme-info">
    主题 -
    <a class="theme-link" href="https://github.com/olOwOlo/hugo-theme-even">Even</a>
  </span>

  <div class="busuanzi-footer">
    <span id="busuanzi_container_site_pv"> 本站总访问量 <span id="busuanzi_value_site_pv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 次 </span>
    <span class="division">|</span>
    <span id="busuanzi_container_site_uv"> 本站总访客数 <span id="busuanzi_value_site_uv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 人 </span>
  </div>

  <span class="copyright-year">
    &copy;
    2020 -
    2021
    <span class="heart">
      <i class="iconfont icon-heart"></i>
    </span>
    <span class="author">早起的年轻人</span>
  </span>
</div>

    </footer>

    <div class="back-to-top" id="back-to-top">
        <i class="iconfont icon-up"></i>
    </div>
</div>

  <script src="https://cdn.jsdelivr.net/npm/jquery@3.2.1/dist/jquery.min.js" integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/slideout@1.0.1/dist/slideout.min.js" integrity="sha256-t+zJ/g8/KXIJMjSVQdnibt4dlaDxc9zXr/9oNPeWqdg=" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@3.1.20/dist/jquery.fancybox.min.js" integrity="sha256-XVLffZaxoWfGUEbdzuLi7pwaUJv1cecsQJQqGLe7axY=" crossorigin="anonymous"></script>
    <script src="https://cdn.jsdelivr.net/npm/timeago.js@3.0.2/dist/timeago.min.js" integrity="sha256-jwCP0NAdCBloaIWTWHmW4i3snUNMHUNO+jr9rYd2iOI=" crossorigin="anonymous"></script>
    <script src="https://cdn.jsdelivr.net/npm/timeago.js@3.0.2/dist/timeago.locales.min.js" integrity="sha256-ZwofwC1Lf/faQCzN7nZtfijVV6hSwxjQMwXL4gn9qU8=" crossorigin="anonymous"></script>
  <script><!-- NOTE: timeago.js uses the language code format like "zh_CN" (underscore and case sensitive) -->
    var languageCode = "zh-CN".replace(/-/g, '_').replace(/_(.*)/, function ($0, $1) {return $0.replace($1, $1.toUpperCase());});
    timeago().render(document.querySelectorAll('.timeago'), languageCode);
    timeago.cancel();  
  </script>
<script type="text/javascript" src="/dist/even.47f727f4.min.js"></script>


<script type="application/javascript">
var doNotTrack = false;
if (!doNotTrack) {
	window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
	ga('create', 'UA-138883536-1', 'auto');
	ga('set', 'anonymizeIp', true);
	ga('send', 'pageview');
}
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>







</body>
</html>
